﻿import typing
import multiprocessing.managers
import multiprocessing.synchronize
import time
import sys

import requestNovelsLib
from concurrent.futures import ThreadPoolExecutor

import datetime
import traceback
import lxml
import lxml.etree
import requests

timeFmt = "%Y-%m-%d %H:%M:%S"

__runMain = False

__maxNovelCount = 250
_pageinfocount = 2000
__var: multiprocessing.managers.ValueProxy[int] = None
__typeCount: multiprocessing.managers.ValueProxy[int] = None
__novelMapCount: multiprocessing.managers.ValueProxy[dict[str, int]] = None


def getNovelHome(response: requests.Response) -> typing.Union[dict[str, list[str]], None]:
	"""
	首页查找到类型与 url 的映射
	:param response: get请求之后而的响应体
	:return: 类型名称与之匹配的 url 映射
	"""
	xpath = requestNovelsLib.XpathHtml(response.text)
	## 获取 li 节点
	liNodes = xpath.xpath(r'//div[@class="hd"]/ul/li/a')
	if xpath.resultLen() == 0:
		return
	## 类型名过滤器
	getTypeName = requestNovelsLib.getNovelTypeName()
	## 返回
	result: dict[str, list[str]] = {}
	## 遍历需要的类型名称
	typeNameUrl = ""
	for liNode in liNodes:
		liNodeXpathInfo = requestNovelsLib.XpathInfo(liNode)
		aTests = liNodeXpathInfo.xpath(r'./text()')
		if liNodeXpathInfo.resultLen() == 0:
			continue
		typeName = aTests[0]
		typeName = requestNovelsLib.strRemoveSpaces(typeName)
		for item in getTypeName:
			if item == typeName:
				xpathList = liNodeXpathInfo.xpath(r'./@href')
				if len(xpathList) == 0:
					break
				href = xpathList[0]
				href = href.replace(response.url, "")[1:]
				typeNameUrl = requestNovelsLib.getResponseAtStr(response, href)
				break
		if len(typeNameUrl) != 0:
			if result.get(typeName) is None:
				result[typeName] = [typeNameUrl]
			else:
				result[typeName].append(typeNameUrl)
			__novelMapCount[typeName] = _pageinfocount
			typeNameUrl = ""
	__var.value = requestNovelsLib.getWorkerThreadNumber()
	__typeCount.value = len(result)
	return result


def getPageListNovel(response: requests.Response, getCount: int, novelTypeName: str, novelTypeUrl: str, novelRootUrl: str) -> typing.Union[list[requestNovelsLib.NovelInfo], None]:
	"""
	获取小说信息
	若为 none 或 数量 0， 则退出这次类型的扫描
	:param response: get 之后的响应体
	:param getCount: 已经库存的数量
	:param novelTypeName: 小说类型名称
	:param novelTypeUrl: 小说类型 url
	:param novelRootUrl: 小说网站
	:return: 小说列表
	"""
	global __maxNovelCount, __runMain, __novelMapCount
	xpath = requestNovelsLib.XpathHtml(response.text)
	liNodes = xpath.xpath(r'//div[@class="cf"]/dl')
	if xpath.resultLen() == 0:
		return None
	## 小说网址
	novelUrl: str = ""
	## 小说名称
	novelName: str = ""
	## 小说作者
	author: str = ""
	## 最后更新日期
	updateTime: str = ""
	## 最后更新选项
	lastItem: str = ""
	## 小说详情
	info: str = ""
	## 返回值
	result: list[requestNovelsLib.NovelInfo] = []
	
	typeGetCount = __novelMapCount[novelTypeName]
	if typeGetCount <= 0:
		return None
	typeGetCount = typeGetCount - len(liNodes)
	if typeGetCount < 0:
		typeGetCount = 0
	__novelMapCount[novelTypeName] = typeGetCount
	# pc 转 移动
	# novelRootUrl = novelRootUrl.replace("https://www.", "https://m.", 1)
	for liNode in liNodes:
		novelUrl = novelName = novelName = author = updateTime = lastItem = info = ""
		try:
			## 获取时间
			nodeXpathInfo = requestNovelsLib.XpathInfo(liNode)
			nodeTextArray = nodeXpathInfo.xpath(r'./dd/h3/span[@class="uptime"]/text()')
			resultLen = nodeXpathInfo.resultLen()
			if resultLen == 0:
				print(f"错误的 => 日期 [{novelTypeName} ({getCount})] error : error ==== >")
				continue
			updateTime = nodeTextArray[0]
			## 时间跳出
			## 2022-09-22 11:42:11
			# if not requestNovelsLib.checkUpDate(updateTime, timeFmt):
			# 	continue
			
			## 获取名称
			nodeTextArray = nodeXpathInfo.xpath(r'./dd/h3/a/text()')
			if nodeXpathInfo.resultLen() == 0:
				print(f"错误的 => 名称 [{novelTypeName} ({getCount})] error : error ==== >")
				continue
			novelName = nodeTextArray[0]
			
			## 获取链接
			nodeTextArray = nodeXpathInfo.xpath(r'./dd/h3/a/@href')
			if nodeXpathInfo.resultLen() == 0:
				print(f"错误的 => 链接 [{novelTypeName} ({getCount})] {novelName} :  error ==== >")
				continue
			novelUrl = f"https://{novelRootUrl}/{nodeTextArray[0][1:]}"
			
			## 获取最后更新项
			nodeTextArray = nodeXpathInfo.xpath(r'./dd[@class="book_other"]/a/text()')
			if nodeXpathInfo.resultLen() == 0:
				continue
			lastItem = nodeTextArray[0]
			
			## 获取作者
			nodeTextArray = nodeXpathInfo.xpath(r'./dd[@class="book_other"]/span/text()')
			if nodeXpathInfo.resultLen() != 0:
				author = nodeTextArray[0]
			else:
				print(f"错误的 => 作者 [{novelTypeName} ({getCount})] {novelName} : {novelUrl} ==== >")
			
			## 获取详情
			nodeTextArray = nodeXpathInfo.xpath(r'./dd[@class="book_des"]/text()')
			if nodeXpathInfo.resultLen() != 0:
				xpathArray = nodeTextArray[0]
				if xpathArray is not None and len(xpathArray) > 0:
					info = xpathArray
			else:
				print(f"错误的 => 详情 [{novelTypeName} ({getCount})] {novelName} : {novelUrl} ==== >")
			
			## 添加单个对象
			result.append(requestNovelsLib.NovelInfo(novelRootUrl=novelRootUrl, novelTypeName=novelTypeName, novelTypeUrl=novelTypeUrl, novelUrl=novelUrl, novelName=novelName, author=author, updateTime=updateTime, lastItem=lastItem, info=info, updateTimeFmt=timeFmt))
			getCount += 1
			if not __runMain:
				if __maxNovelCount is None:
					__maxNovelCount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "count", __defMaxNovelCount))
				if 0 < __maxNovelCount < getCount:
					break
		except BaseException as es:
			traceback.print_exc()
			print(f"================>\n\t{novelUrl}\n<===========")
	print(f"https://{requestNovelsLib.findUrlName(novelRootUrl)} : {novelTypeName}类型计数 :  {typeGetCount}")
	## 返回
	if len(result) == 0:
		return None
	return result


def getAppendRequestNovelPageFunctionPtr(novelInfo: requestNovelsLib.NovelInfo, response: requests.Response, requestGetUrlDate: datetime, getCount=0) -> bool:
	"""
	决定是否调用 appRequestNovelPage， 返回 appRequestNovelPage 表示继续调用
	:param novelInfo: 当前的小说信息
	:param response: 当前的小说信息来源
	:param requestGetUrlDate:  请求时间
	:param getCount: 获取的数量
	:return: 一个 appRequestNovelPage 的调用
	"""
	global __nextRequeInfoSec, __var, __maxNovelCount
	if requestNovelsLib.checkUpDate(novelInfo.updateTime, novelInfo.updateTimeFmt):
		print(f"存档 ({novelInfo.novelName}) [{novelInfo.updateTime}] : {novelInfo.novelUrl} [{novelInfo.novelTypeName}] : {getCount + 1} / {__maxNovelCount}")
		return None
	return appRequestNovelPage


__nextRequeInfoSec = 0.8


def appRequestNovelPage(novelInfo: requestNovelsLib.NovelInfo, response: requests.Response, requestGetUrlDate: datetime, getCount=0) -> requestNovelsLib.NovelInfo:
	"""
	用于请求小说页面
	:param novelInfo: 小说内容
	:param response: 小说详细页面
	:param requestGetUrlDate: 小说请求时候的时间
	:param getCount: 当前有效个数
	:return: 完整小说
	"""
	global __nextRequeInfoSec, __var, __maxNovelCount
	if response is None:
		return None
	try:
		
		## 获取小说内容
		xpath = requestNovelsLib.XpathHtml(requestNovelsLib.getAutoCodeText(response))
		liNodes = xpath.xpath(r'//p[@class="book-stats"]')
		if xpath.resultLen() == 0:
			return None
		for pNode in liNodes:
			## 子节点
			nodeXpathInfo = requestNovelsLib.XpathInfo(pNode)
			texts = nodeXpathInfo.xpath(r'./b/text()')
			if nodeXpathInfo.resultLen() == 0:
				continue
			text = texts[-1].replace("更新时间：", "")
			if requestNovelsLib.dateTimeComp(text, novelInfo.updateTime, novelInfo.updateTimeFmt) > 0:
				novelInfo.updateTime = text
			if not requestNovelsLib.checkUpDate(novelInfo.updateTime, novelInfo.updateTimeFmt):
				# print(f"({novelInfo.novelTypeName})[{novelInfo.novelName}]已经过期 {novelInfo.updateTime}:({novelInfo.novelUrl})")
				return None
			novelInfo.updateTime = text
		print(f"存档 ({novelInfo.novelName}) [{novelInfo.updateTime}] : {novelInfo.novelUrl} [{novelInfo.novelTypeName}] : {getCount + 1} / {__maxNovelCount}")
		return novelInfo
	except:
		requestNovelsLib.writeLogFile("小说获取异常", novelInfo.novelRootUrl)
		sys.stderr.write(traceback.format_exc())
	finally:  ## 比较时间
		currentDate = nodeBuffStr = datetime.datetime.now()
		modDate = currentDate - requestGetUrlDate
		sec: int = int(modDate.total_seconds() + 0.91)
		sleepSec = __nextRequeInfoSec * __var.value
		if sec < sleepSec:
			sec = sleepSec - sec
			time.sleep(sec)
	return None


def getPageNextUrl(response: requests.Response, exsiNovel: int, noveList, appenLen, novelTypeName: str, novelTypeUrl: str, novelRootUrl: str) -> typing.Union[str, None]:
	"""
	获取小说翻页的下一页
	:param response: get 之后的响应体
	:param exsiNovel: 已经库存的小说
	:param noveList: 查找到的小说
	:param appenLen: 新增数量
	:param novelTypeName: 小说类型名称
	:param novelTypeUrl: 小说类型 url
	:param novelRootUrl: 小说网站
	:return: 下一页，不存在则返回 none
	"""
	if appenLen == 0:
		return None
	try:
		xpath = requestNovelsLib.XpathHtml(response.text)
		pageList = xpath.xpath(r'//div[@class="pages"]/ul')
		if xpath.resultLen() == 0:
			return None
		divNodes = requestNovelsLib.XpathInfo(pageList[0])
		currentXpathNode = divNodes.xpath(r'./li[@class="current"]/a/@href')
		if divNodes.resultLen() == 0:
			return None
		currentHtml = currentXpathNode[0]
		
		node_xpath = divNodes.xpath(r'./li/a/@href')
		
		if divNodes.resultLen() == 0:
			return None
		lastNode = node_xpath[-1]
		if currentHtml == lastNode:
			return None
		resultNextPageUrl = f"https://{novelRootUrl}{lastNode}"
		return resultNextPageUrl
	
	except BaseException as es:
		print(f"\n\t====>\n{es}\n\t====>\n")
	return None


def sort(novel: requestNovelsLib.NovelInfo):
	return novel.updateTime


def endCall(novelTypeName: str, novelTypeUrl: str, novelRootUrl: str, requestUrl: str, isAppendNovel: bool):
	"""
	类型调用结束时候被自动调用
	:param novelTypeName: 类型名称
	:param novelTypeUrl: 类型 url
	:param novelRootUrl: 网站 root
	:param requestUrl: 最后请求 url
	:param isAppendNovel: 追加状态识别码
	"""
	global __var, __typeCount, __novelMapCount
	__typeCount.value -= 1
	if __typeCount.value < __var.value:
		__var.value = __typeCount.value
	__novelMapCount[novelTypeName] = 0


__runTime = None


def setRunTime(time):
	global __runTime
	__runTime = time


def getRunTime():
	global __runTime
	return __runTime


def getRootUrl():
	return 'www.121ds.cc'


def runTimeUserCallPtr():
	global __novelMapCount
	
	msg = ""
	try:
		msgLen = False
		for typeName, count in __novelMapCount.items():
			if not msgLen:
				msgLen = True
			else:
				msg += ", "
			msg = f"{msg}{typeName} : {count}"
		msg = f"\n===========\n\thttps://{requestNovelsLib.findUrlName(getRootUrl())}\t{msg}\n===========\n"
		print(msg)
	except:
		exc_type, exc_value, exc_traceback = sys.exc_info()
		errorList = traceback.format_tb(exc_traceback)
		writeContent = ""
		if errorList is not None and len(errorList) != 0:
			writeContent = errorList[0]
		writeContent = f"<!-- 类型计数映射读取出现问题\n{writeContent} -->\n\n\n<!-- {traceback.format_exc()} -->"
		writeLogFile(writeContent, rootUrl)
		sys.stderr.write(writeContent)


def modeRequestGetUrl(lock: multiprocessing.synchronize.Lock = None, workerThreadNumber=0, nextRequestTimeSece=0.5, titleSleepTime=10):
	global __maxNovelCount, __runMain, __var, __typeCount, __novelMapCount, _pageinfocount
	if not __runMain:
		if lock is None:
			raise RuntimeError("请在调用时候设置它的进程锁")
	else:
		if lock is None:
			lock = multiprocessing.Lock()
	
	__var = multiprocessing.Value('i', 0)
	__typeCount = multiprocessing.Value('i', 0)
	manager = multiprocessing.Manager()
	__novelMapCount = manager.dict()
	try:
		__maxNovelCount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "count", __maxNovelCount))
	except:
		requestNovelsLib.writeLogFile("无法获取匹配的数量", httpUrl=getRootUrl())
		traceback.print_exc()
	try:
		_pageinfocount = int(requestNovelsLib.readIni(requestNovelsLib.getDownLoadIniFilePath(), f"{getRootUrl()}", "pageinfocount", _pageinfocount))
	except:
		requestNovelsLib.writeLogFile("无法获取匹配的数量", httpUrl=getRootUrl())
		traceback.print_exc()
	rootUrl = getRootUrl()
	requestUrl = requestNovelsLib.ResponseNovel(f'https://{rootUrl}/')
	newFolder = requestNovelsLib.getUrlTargetFolder(rootUrl)
	try:
		## 获取类型类型的请求页面
		runEndTime = requestNovelsLib.ayscRequestGetUrl(requestUrl, getNovelHome, getPageListNovel, getPageNextUrl, rootUrl, newFolder, __runMain, proLock=lock, nextRequestTimeSece=nextRequestTimeSece, workerThreadNumber=workerThreadNumber, titleSleepTime=titleSleepTime, getAppendRequestNovelPageFunctionPtr=getAppendRequestNovelPageFunctionPtr, endCall=endCall, novelMaxCount=__maxNovelCount, runTimeUserCallPtr=runTimeUserCallPtr)
		setRunTime(runEndTime)
	except:
		traceback.print_exc()
	return


if __name__ == '__main__':
	__runMain = True
	requestNovelsLib.initLib()
	modeRequestGetUrl()
